from __future__ import division
import random
import gzip
from operator import itemgetter, attrgetter
from HTMLParser import HTMLParser
from HTMLParser import HTMLParseError
from htmlentitydefs import name2codepoint
import os
import math
import sys

print "Program Begins..."
print "Updated by Wei 2013/07/13 night"
print "for the feature: posting_rank_in_the_doc, it is handled by the polyIRToolkit again. Python script lose the control."
# METHOD1:
'''
# 1st level
# key: trecID
# value: another dict
    # 2ed level
    # key: term
    # value: posting_rank_in_doc
docDictWithTermDict = {}

inputFileName = "/data3/obukai/workspace/web-search-engine-wei/gov2DocumentWithTheirForwardIndex_Combine_final_20130612_sortedByExternalTrecID"
inputFileHandler = open(inputFileName,"r")
# ignore the headerLine
# headerLine = inputFileHandler.readline()
# print "headerLine:",headerLine

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    externalTrecID = lineElements[0]
    docWords = int( lineElements[1] )
    docPostingRecorded = int( lineElements[2] )
    currentDocPostingsList = []
    
    if externalTrecID not in docDictWithTermDict:
        docDictWithTermDict[externalTrecID] = {}
    else:
        print "mark1"
        exit(1)

    # checking mechanism
    if docPostingRecorded == len(lineElements[3:]):
        pass
    else:
        print "docPostingRecorded:",docPostingRecorded,type(docPostingRecorded)
        print "len(lineElements[2:]):",len(lineElements[3:]),type( len(lineElements[3:]) )
        exit(1)
    
    # loading mechanism
    for posingInfoTupleInStringFormat in lineElements[3:]:
        # print "posingInfoTupleInStringFormat:",posingInfoTupleInStringFormat,type(posingInfoTupleInStringFormat)
        tupleElements = posingInfoTupleInStringFormat.split("(")[1].split(")")[0].split(",")
        term = tupleElements[0]
        partialBM25InFloatFormat = float(tupleElements[1])
        currentDocPostingsList.append( (term,partialBM25InFloatFormat) )
            
    currentDocPostingsList.sort(cmp=None, key=itemgetter(1), reverse=True)
    
    
    # checking mechanism
    # if externalTrecID == "GX000-01-8658041":
    #    print "currentDocPostingsList:",currentDocPostingsList
    
    
    for index,postingTuple in enumerate(currentDocPostingsList):
        (term,_) = postingTuple
        docDictWithTermDict[externalTrecID][term] = index + 1
    
    
    # checking mechanism
    # if externalTrecID == "GX000-01-8658041":
    #    print "docDictWithTermDict[externalTrecID]:",docDictWithTermDict[externalTrecID]
    
    
    
    # check point
    # print "len(docDictWithTermDict[",externalTrecID,"]):",len(docDictWithTermDict[externalTrecID])
    # if len(docDictWithTermDict) % 10000 == 0:
    #    print "len(docDictWithTermDict):",len(docDictWithTermDict)
    #    break
    

print "len(docDictWithTermDict):",len(docDictWithTermDict)
inputFileHandler.close()



# use the posting rank provided in the dict to assign the posting_rank_in_doc for the posting
# option1: for pangolin
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/Posting_Oriented_Training_Dataset_2013_06_09.smallTest.Combine.train_tfqAdded_labelsAdded_rankInListAdded_sortedByExternalTrecID"
# option2: maybe for dodo
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/Posting_Oriented_Training_Dataset_2013_06_09.smallTest.Combine.train_tfqAdded_labelsAdded_rankInListAdded"
inputFileHandler = open(inputFileName,"r")

outputFileName = inputFileName + "_rankInDocAdded"
outputFileHandler = open(outputFileName,"w")

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    externalTrecID = lineElements[2]
    term = lineElements[4]
    # test for the docDictWithTermDict
    if externalTrecID in docDictWithTermDict:
        if term in docDictWithTermDict[externalTrecID]:
            outputLine = line.strip() + " " + str( docDictWithTermDict[externalTrecID][term] ) + "\n"
        else:
            print "term:",term,"is NOT provided in the externalTrecID:",externalTrecID
            outputLine = line.strip() + " " + "N/A" + "\n"
    else:
        print "externalTrecID:",externalTrecID,"is NOT provided."
        outputLine = line.strip() + " " + "N/A" + "\n"
    
    outputFileHandler.write(outputLine)
    
inputFileHandler.close()
outputFileHandler.close()
'''

'''
# METHOD2(currently. Part1/2 ):
inputFileName = "/data3/obukai/workspace/web-search-engine-wei/gov2DocumentWithTheirForwardIndex_Combine_final_20130612_sortedByExternalTrecID"
inputFileHandler1 = open(inputFileName,"r")

# use the posting rank provided in the dict to assign the posting_rank_in_doc for the posting
# option1: for pangolin
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/Posting_Oriented_Training_Dataset_2013_06_09.smallTest.Combine.train_tfqAdded_labelsAdded_rankInListAdded_sortedByExternalTrecID"
# option2: maybe for dodo
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/Posting_Oriented_Training_Dataset_2013_06_09.smallTest.Combine.train_tfqAdded_labelsAdded_rankInListAdded"
inputFileHandler2 = open(inputFileName,"r")

outputFileName = inputFileName + "_rankInDocAdded_using_Combine_final_20130612"
outputFileHandler = open(outputFileName,"w")

outputAuxFileName = "/data3/obukai/workspace/web-search-engine-wei/document_set_needed_to_be_REPARSED_again_for_the_1_term_query_problem_NEW"
outputAuxFileHandler = open(outputAuxFileName,"w")

currentDocTermRankingDict = {}
currentRecordedTrecID = ""
for line in inputFileHandler2.readlines():
    lineElements = line.strip().split(" ")
    queryID = lineElements[1]
    externalTrecID = lineElements[2]
    queryTerm = lineElements[4]
    
    if currentRecordedTrecID != externalTrecID: # need to load the info one more line
        lineElementsForPostingsInfo = inputFileHandler1.readline().strip().split(" ")
        externalTrecIDFromPostingsInfoFile = lineElementsForPostingsInfo[0]
        if externalTrecID == externalTrecIDFromPostingsInfoFile:
            # print "TrecID is the same"
            currentRecordedTrecID = externalTrecID
            # Do the real logic of assigning
            docWords = int( lineElementsForPostingsInfo[1] )
            docPostingRecorded = int( lineElementsForPostingsInfo[2] )
            currentDocPostingsList = []
            currentDocTermRankingDict = {}
            # checking mechanism
            if docPostingRecorded == len(lineElementsForPostingsInfo[3:]):
                pass
            else:
                print "docPostingRecorded:",docPostingRecorded,type(docPostingRecorded)
                print "len( lineElementsForPostingsInfo[3:] ):",len( lineElementsForPostingsInfo[3:] ),type( len( lineElementsForPostingsInfo[3:] ) )
                exit(1)
            
            # loading mechanism
            for posingInfoTupleInStringFormat in lineElementsForPostingsInfo[3:]:
                # print "posingInfoTupleInStringFormat:",posingInfoTupleInStringFormat,type(posingInfoTupleInStringFormat)
                tupleElements = posingInfoTupleInStringFormat.split("(")[1].split(")")[0].split(",")
                termFromPostingInfo = tupleElements[0]
                partialBM25InFloatFormat = float(tupleElements[1])
                currentDocPostingsList.append( (termFromPostingInfo,partialBM25InFloatFormat) )
                    
            currentDocPostingsList.sort(cmp=None, key=itemgetter(1), reverse=True)
            
            for index,postingTuple in enumerate(currentDocPostingsList):
                (termFromPostingInfo,_) = postingTuple
                currentDocTermRankingDict[termFromPostingInfo] = index + 1
    else:
        # No need to reload the posting info, and just use the existing currentDocTermRankingDict and will be ok
        pass
    
    if queryTerm in currentDocTermRankingDict:
        outputLine = line.strip() + " " + str( currentDocTermRankingDict[queryTerm] ) + "\n"
    else:
        print "missing:",queryID,queryTerm,externalTrecID
        outputLine = line.strip() + " " + "N/A" + "\n"
        outputAuxFileHandler.write(queryID + " " + queryTerm + " " + externalTrecID + "\n")
    
    outputFileHandler.write(outputLine)

inputFileHandler1.close()   
inputFileHandler2.close()
outputAuxFileHandler.close()
outputFileHandler.close()
'''

'''
# METHOD2(continued. Part2/2 ):
###############################
# 1st level
# key: trecID
# value: another dict
    # 2ed level
    # key: term
    # value: posting_rank_in_doc
docDictWithTermDict = {}

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/gov2DocumentWithTheirForwardIndex_the_rest_of_missing_posting_documents_534"
inputFileHandler = open(inputFileName,"r")
# ignore the headerLine
# headerLine = inputFileHandler.readline()
# print "headerLine:",headerLine

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    externalTrecID = lineElements[0]
    docWords = int( lineElements[1] )
    docPostingRecorded = int( lineElements[2] )
    currentDocPostingsList = []
    
    if externalTrecID not in docDictWithTermDict:
        docDictWithTermDict[externalTrecID] = {}
    else:
        print "mark1"
        exit(1)

    # checking mechanism
    if docPostingRecorded == len(lineElements[3:]):
        pass
    else:
        print "docPostingRecorded:",docPostingRecorded,type(docPostingRecorded)
        print "len(lineElements[2:]):",len(lineElements[3:]),type( len(lineElements[3:]) )
        exit(1)
    
    # loading mechanism
    for posingInfoTupleInStringFormat in lineElements[3:]:
        # print "posingInfoTupleInStringFormat:",posingInfoTupleInStringFormat,type(posingInfoTupleInStringFormat)
        tupleElements = posingInfoTupleInStringFormat.split("(")[1].split(")")[0].split(",")
        term = tupleElements[0]
        partialBM25InFloatFormat = float(tupleElements[1])
        currentDocPostingsList.append( (term,partialBM25InFloatFormat) )
            
    currentDocPostingsList.sort(cmp=None, key=itemgetter(1), reverse=True)
    
    
    # checking mechanism
    # if externalTrecID == "GX000-01-8658041":
    #    print "currentDocPostingsList:",currentDocPostingsList
    
    
    for index,postingTuple in enumerate(currentDocPostingsList):
        (term,_) = postingTuple
        docDictWithTermDict[externalTrecID][term] = index + 1
    
    
    # checking mechanism
    # if externalTrecID == "GX000-01-8658041":
    #    print "docDictWithTermDict[externalTrecID]:",docDictWithTermDict[externalTrecID]
    
    
    
    # check point
    # print "len(docDictWithTermDict[",externalTrecID,"]):",len(docDictWithTermDict[externalTrecID])
    # if len(docDictWithTermDict) % 10000 == 0:
    #    print "len(docDictWithTermDict):",len(docDictWithTermDict)
    #    break
    

print "len(docDictWithTermDict):",len(docDictWithTermDict)
inputFileHandler.close()
###############################

# docsDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/Posting_Oriented_Training_Dataset_2013_06_09.smallTest.Combine.train_tfqAdded_labelsAdded_rankInListAdded_sortedByExternalTrecID_rankInDocAdded_using_Combine_final_20130612"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/Posting_Oriented_Training_Dataset_2013_06_09.smallTest.Combine.train_tfqAdded_labelsAdded_rankInListAdded_sortedByExternalTrecID_rankInDocAdded_using_Combine_final_20130612_Combine_postings_missing_document_set"
outputFileHandler = open(outputFileName,"w")

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/the_lines_fixing_the_missing_posting_document"
outputFileHandler2 = open(outputFileName,"w")

missingAssignmentNumber = 0
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    rank = lineElements[-1]
    if rank == "N/A":
        externalTrecID = lineElements[2]
        queryTerm = lineElements[4]
        if externalTrecID in docDictWithTermDict:
            if queryTerm in docDictWithTermDict[externalTrecID]:
                rank = docDictWithTermDict[externalTrecID][queryTerm]
                newOutputLine = ""
                for element in lineElements[:-1]:
                    newOutputLine += element + " "
                newOutputLine += str(rank)
                newOutputLine += "\n"
                print newOutputLine,
                outputFileHandler.write(newOutputLine)
                outputFileHandler2.write(newOutputLine)
            else:
                outputFileHandler.write(line)
                missingAssignmentNumber += 1
        else:
            outputFileHandler.write(line)
            missingAssignmentNumber += 1
        # if externalTrecID not in docsDict:
        #    docsDict[externalTrecID] = 1
        
    else:
        outputFileHandler.write(line)

print "missingAssignmentNumber:",missingAssignmentNumber
# print "len(docsDict):",len(docsDict)        
# print "docsDict:",docsDict

inputFileHandler.close()
outputFileHandler.close()
outputFileHandler2.close()
'''


print "Program Ends."
















